{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.spatial.transform import Rotation as R\n",
    "from astropy import coordinates as ac\n",
    "import copy\n",
    "import torch\n",
    "\n",
    "import os \n",
    "from os.path import join, split,exists, isdir,isfile\n",
    "\n",
    "file_root =  join(os.getcwd(),'..')\n",
    "os.chdir(file_root)\n",
    "\n",
    "from utils.pc_utils import *\n",
    "from utils import * \n",
    "\n",
    "\n",
    "from utils.utils import * \n",
    "import numpy as np \n",
    "\n",
    "\n",
    "from entity.prediction_loader import PredictionLoader\n",
    "\n",
    "from entity.semantic_kitti_gt_loader import SementicKittiGtLoader\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from entity.semantic_evaluator import SementicEvaluator\n",
    "\n",
    "import time \n",
    "from tqdm import tqdm"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## init the  evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_epoch_0\n",
      "model_epoch_1\n",
      "model_epoch_2\n",
      "model_epoch_3\n",
      "model_epoch_4\n",
      "model_epoch_5\n",
      "model_epoch_6\n",
      "model_epoch_7\n",
      "model_epoch_8\n",
      "model_epoch_9\n",
      "model_epoch_10\n",
      "model_epoch_11\n",
      "model_epoch_12\n",
      "model_epoch_13\n",
      "model_epoch_14\n",
      "model_epoch_15\n",
      "model_epoch_16\n",
      "model_epoch_17\n",
      "model_epoch_18\n",
      "model_epoch_19\n",
      "model_epoch_20\n",
      "model_epoch_21\n",
      "model_epoch_22\n",
      "model_epoch_23\n",
      "model_epoch_24\n",
      "model_epoch_25\n",
      "model_epoch_26\n",
      "model_epoch_27\n",
      "model_epoch_28\n",
      "model_epoch_29\n",
      "model_epoch_30\n",
      "model_epoch_31\n",
      "model_epoch_32\n",
      "model_epoch_33\n",
      "model_epoch_34\n",
      "model_epoch_35\n",
      "model_epoch_36\n",
      "model_epoch_37\n",
      "model_epoch_38\n",
      "model_epoch_39\n",
      "model_epoch_40\n",
      "model_epoch_41\n",
      "model_epoch_42\n",
      "model_epoch_43\n",
      "model_epoch_44\n",
      "model_epoch_45\n",
      "model_epoch_46\n",
      "model_epoch_47\n",
      "model_epoch_48\n",
      "model_epoch_49\n",
      "total spend  time  :  00:18:38\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "prediction_root = \"datasets/predictions/Jan19_four_losses_with_shapenet_anomaly\"\n",
    "data_path = 'datasets/dataset'\n",
    "\n",
    "model_pathes = sorted([x for x in os.listdir(prediction_root)],key=lambda x:int(x.split('_')[-1]))\n",
    "\n",
    "evaluator_list = []\n",
    "tic = time.time()\n",
    "for path in tqdm(model_pathes):\n",
    "    evaluator_list.append(SementicEvaluator(data_path,join(prediction_root,path),'config/semantic-kitti.yaml',split='valid'))\n",
    "    print(path)\n",
    "\n",
    "print('total spend  time  : ',time.strftime(\"%H:%M:%S\",time.gmtime(time.time() - tic)))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## retrieve the best by relative path "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def get_anomaly_matrics(path):\n",
    "    with open(path,'r') as f:\n",
    "        data = json.load(f)\n",
    "    return data \n",
    "\n",
    "\n",
    "\n",
    "def get_best_metrics(prediction_root,model_pathes):\n",
    "    best_AUPR = best_AUROC = 0\n",
    "    best_FPR95=1e+2\n",
    "    best_AUPR_epoch = best_AUROC_epoch =  best_FPR95_epoch = None\n",
    "    error_model_name_list = []\n",
    "    for model_name in model_pathes:\n",
    "        # print(model_name)\n",
    "        try:\n",
    "            prediciton_model_path = join(prediction_root,model_name,'anomaly_eval_results.json')\n",
    "            data = get_anomaly_matrics(prediciton_model_path)\n",
    "        except Exception as e :\n",
    "            error_model_name_list.append(model_name)\n",
    "            print(e)\n",
    "            continue\n",
    "\n",
    "\n",
    "        if data['OOD/AUPR'] > best_AUPR:\n",
    "            best_AUPR = data['OOD/AUPR']\n",
    "            best_AUPR_epoch = model_name\n",
    "\n",
    "        if data['OOD/AUROC'] > best_AUROC:\n",
    "            best_AUROC = data['OOD/AUROC'] \n",
    "            best_AUROC_epoch = model_name\n",
    "        \n",
    "        if data['OOD/FPR95'] <  best_FPR95:\n",
    "            best_FPR95 = data['OOD/FPR95'] \n",
    "            best_FPR95_epoch = model_name\n",
    "    return best_AUPR,best_AUPR_epoch,best_AUROC,best_AUROC_epoch,best_FPR95,best_FPR95_epoch\n",
    "\n",
    "tic = time.time()\n",
    "\n",
    "\n",
    "prediction_root = \"datasets/predictions/Jan19_four_losses_with_shapenet_anomaly\"\n",
    "model_pathes = sorted([x for x in os.listdir(prediction_root) if x.startswith('model')],key=lambda x:int(x.split('_')[-1]))\n",
    "best_AUPR,best_AUPR_epoch,best_AUROC,best_AUROC_epoch,best_FPR95,best_FPR95_epoch = get_best_metrics(prediction_root,model_pathes)\n",
    "print('total spend  time  : ',time.strftime(\"%H:%M:%S\",time.gmtime(time.time() - tic)))\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_AUPR,best_AUPR_epoch,best_AUROC,best_AUROC_epoch,best_FPR95,best_FPR95_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = get_anomaly_matrics(join(prediction_root,'model_epoch_14','anomaly_eval_results.json'))\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "yang_real",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
